package engine

import (
	"goruntimeCrawler/fetcher"
	"log"
)

type SimpleEngine struct {

}

func (s SimpleEngine) Run(seeds ...Request)  {
	var requests []Request
    for _,i:=range seeds{
    	//log.Printf("got url:%s",i.UrL)
    	requests=append(requests,i)
	}
    for len(requests)>0 {
    	r:=requests[0]
    	requests=requests[1:]

		parseResult,err:=s.worker(r)
		if err!=nil {
			continue
		}

        requests=append(requests,parseResult.Requests...)

		for _,item:=range parseResult.Items{
			log.Printf("Got item:%v",item)
		}
	}
}

func (SimpleEngine) worker(r Request) (ParseResult,error) {
	body,err:=fetcher.Fetch(r.UrL)
	if err !=nil {
		log.Printf("Fetcher error url:%s, eror:%v",r.UrL,err)
		return ParseResult{},err
	}
	return r.ParserFunc(body),nil
}




